# -*- coding: UTF-8 -*-
# cython: language_level=3
import re
import socket
from concurrent import futures

import requests
from lxml import etree
from requests.utils import get_encodings_from_content
from tqdm import tqdm

from db.mongodb import MongoDB
from db.oracle_db import OracleDB
from utils.ip_area import find_ip
from utils.own_tools import fake_ua, raise_error_site_name

socket.setdefaulttimeout(3.1)

db = MongoDB()
orcl_db = OracleDB()


def save_icp(oid, NEW_SITE_NAME='',
             NEW_AUTO_RECORD_NUMBER='',
             NEW_IP_ADDRESS='', VISIT_STATUS=0):
    try:
        assert NEW_IP_ADDRESS
        country, PROVINCE_NAME, CITY_NAME, isp = find_ip(NEW_IP_ADDRESS)
        if country and country != '中国':
            PROVINCE_NAME = country + PROVINCE_NAME
    except:
        PROVINCE_NAME = ''
        CITY_NAME = ''
    NEW_SITE_NAME = NEW_SITE_NAME or ''
    NEW_SITE_NAME = str(NEW_SITE_NAME).replace("'", '')[:100] or '无'
    NEW_AUTO_RECORD_NUMBER = NEW_AUTO_RECORD_NUMBER or ''
    NEW_AUTO_RECORD_NUMBER = NEW_AUTO_RECORD_NUMBER.replace("'", '')
    sql = f"update TAB_VIDEO_SITE_INFO " \
          f"set VISIT_STATUS={VISIT_STATUS}," \
          f"NEW_SITE_NAME='{NEW_SITE_NAME}'," \
          f"NEW_AUTO_RECORD_NUMBER='{NEW_AUTO_RECORD_NUMBER}'," + \
          (f"CHECK_RECORD_NUMBER='{NEW_AUTO_RECORD_NUMBER}', " if NEW_AUTO_RECORD_NUMBER else '') + \
          f"NEW_IP_ADDRESS='{NEW_IP_ADDRESS}'," \
          f"PROVINCE_NAME='{PROVINCE_NAME}'," + \
          f"CITY_NAME='{CITY_NAME}'" + \
          (",NEW_FIND_DATE=SYSDATE " if VISIT_STATUS == 0 else '') + f"where id={oid}"
    r = orcl_db.update(sql)
    print(r, sql)


def add_root_url(home_page, host, oid):
    fake_headers = {'User-Agent': fake_ua.random}
    NEW_SITE_NAME = ''
    # NEW_AUTO_RECORD_NUMBER = ''
    NEW_IP_ADDRESS = ''

    try:
        html = requests.get(home_page, headers=fake_headers, timeout=(3.05, 5), stream=True, verify=False)
        html.raise_for_status()

        try:
            NEW_IP_ADDRESS = html.raw._connection.sock.getpeername()[0]
        except:
            print('request erro')
            try:
                NEW_IP_ADDRESS = socket.getaddrinfo(home_page.split('://')[-1].split('/')[0], None)[0][4][0]
            except:
                pass

        if not html.encoding or html.encoding == 'ISO-8859-1':
            encodings = get_encodings_from_content(html.text)
            encodings = encodings[0] if encodings else html.apparent_encoding

            if (not encodings) or (encodings.lower() not in ['gbk', 'utf-8', 'utf8', 'gb2312']):
                html.encoding = 'gbk'
            else:
                html.encoding = encodings
        htmlstr = html.text
        html = etree.HTML(html.text)

        title = html.xpath('//title/text()')
        title = title[0].strip() if title else ''
        NEW_SITE_NAME = title

        icp = re.search('([\u4e00-\u9fa5]\s*ICP\s*[\u4e00-\u9fa5]*\s*\d{4,12}\s*号*-*\d*)', htmlstr)
        if icp:
            NEW_AUTO_RECORD_NUMBER = icp.group(1)
        else:
            NEW_AUTO_RECORD_NUMBER = ''
        print(NEW_AUTO_RECORD_NUMBER, NEW_SITE_NAME)
        # video_license = re.search(
        #     '((信息网络传播视听节目许可证|信息网络传播视听许可证|视听许可证).{0,4}\d{0,12}号*)', htmlstr)
        # video_license = video_license.group(1) if video_license  else ''
        try:
            raise_error_site_name(title)
        except Exception as e:
            VISIT_STATUS = 1
        else:
            VISIT_STATUS = 0

        save_icp(oid, NEW_SITE_NAME, NEW_AUTO_RECORD_NUMBER, NEW_IP_ADDRESS, VISIT_STATUS=VISIT_STATUS)


    except:
        try:
            NEW_SITE_NAME = html.status_code
        except:
            pass
        print(404)
        NEW_AUTO_RECORD_NUMBER = ''
        save_icp(oid, NEW_SITE_NAME, NEW_AUTO_RECORD_NUMBER, NEW_IP_ADDRESS, VISIT_STATUS=1)


def check_main(only_null=False):
    sql = "select ID,URL,DOMAIN from TAB_VIDEO_SITE_INFO where site_type=3"
    if only_null:
        sql = sql + ' and NEW_SITE_NAME is null'
    INPUT_URLS = orcl_db.find(sql)
    print('添加url数量： ', len(INPUT_URLS))
    p_num = min(len(INPUT_URLS), 60)
    if not p_num:
        return
    # for oid, url, host in INPUT_URLS:
    #     add_root_url(url, host, oid)

    tasks, results = [], []
    with futures.ProcessPoolExecutor(max_workers=p_num) as executor:
        for oid, url, host in INPUT_URLS:
            tasks.append(executor.submit(add_root_url, url, host, oid))
        for task in tqdm(futures.as_completed(tasks), total=len(tasks), desc='update_icp'):
            # results.append(task.result())
            pass
    # print(results)


if __name__ == "__main__":
    # check_main(only_null=True)
    check_main()

    print('start screenshot...')

    from utils.screenshot import screen_shot

    screen_shot()
